/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package amd.core.extraction;

import amd.core.data.ListTermFrequency;
import amd.core.*;
import java.util.StringTokenizer;

/**
 *
 * @author Latifah Dwi Nurachma
 */
public class Tokenizer {
    private static String punctuation[] = {".", ",", ";", ":", "(", ")", "-", "/", "?", "!", "\"", "\'", "[", "]", "“", "”","‘", "{", "}"," " };


    public static ListTermFrequency tokenizing(String stringInput){
        ListTermFrequency terms = new ListTermFrequency();
        StringTokenizer stringTokenizer = new StringTokenizer(stringInput.toLowerCase());
        while(stringTokenizer.hasMoreTokens()){
            String newString = stringTokenizer.nextToken();
            while(!stringInput.isEmpty() && startsWithPunctuation(newString)){
                newString = newString.substring(1);
            }
            while (!stringInput.isEmpty() && endsWithPunctuation(newString)){
                newString = newString.substring(0, newString.length()-1);
            }
            newString = newString.trim();
            if(!stringInput.isEmpty()){
                if(newString.contains("http") || newString.contains("www") || newString.contains("@"))
                    System.out.println("...diabaikan...");
                else
                    terms.add(newString);
            }
        }
        return terms;
    }

    private static boolean startsWithPunctuation(String s){
        for (int i =0; i< punctuation.length; i++){
            if (s.startsWith(punctuation[i])){
                return true;
            }
        }
        return false;
    }

    private static boolean endsWithPunctuation(String s){
        for (int i =0; i< punctuation.length; i++){
            if (s.endsWith(punctuation[i])){
                return true;
            }
        }
        return false;
    }

    private static boolean isNumber(char charInput){
		return (charInput=='0'||charInput=='1'||charInput=='2'||charInput=='3'||charInput=='4'||charInput=='5'||charInput=='6'||charInput=='7'||charInput=='8'||charInput=='9');
	}

	/**
	 *
	 * @param charInput
	 */

	private static boolean isPunctuation(String s){
        int i=0;
        while(i<punctuation.length){
            if(s.equalsIgnoreCase(punctuation[i])){
                return true;
            }
            else{
                i++;
            }
        }
        return false;
//        return((charInput==' ')||(charInput=='!')||(charInput=='\"')
//                ||(charInput=='(')||(charInput==')')||(charInput==',')||(charInput=='-')
//                ||(charInput=='.')||(charInput=='/')||(charInput==':')||(charInput==';')
//                ||(charInput=='?')||(charInput=='\n')||(charInput=='\r')||(charInput=='\t'));
	}


}
